# -*- coding: utf-8 -*-

# Scrapy settings for qihu_search project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
from qihu_search.user_agents import AGENTS

BOT_NAME = 'qihu_search'

SPIDER_MODULES = ['qihu_search.spiders']
NEWSPIDER_MODULE = 'qihu_search.spiders'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False
LOG_LEVEL = 'WARNING'
LOG_FILE = './log.log'

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

DOWNLOAD_DELAY = 0.3
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
# COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
# DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
# }

# Enable or disable spider middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
#    'qihu_search.middlewares.QihuSearchSpiderMiddleware': 543,
# }

# Enable or disable downloader middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   'qihu_search.middlewares.RandomUserAgent': 1,
    'scrapy.downloadermiddleware.httpproxy.HttpProxyMiddleware': None,
    'qihu_search.middlewares.ProxyMiddleware': 125,
}

# Enable or disable extensions
# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
# EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
# }

# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'qihu_search.pipelines.QihuSearchPipeline': 300,
}

# 设置为广度优先
DEPTH_PRIORITY = 1
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'

# redis的相关配置，以下配置后，spider无需继承RedisSpider也可将requests和dupefilter存到redis中
# 当requests全部发出去后，删掉对应的key
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
SCHEDULER_PERSIST = True
REDIS_URL = "redis://123.160.246.41:6380"
# REDIS_URL = "redis://192.168.3.191:6379"
REDIS_PARAMS = {'db': 2}  # redis的几号数据库，6层用的是3号库

REDIS_HOST = '123.160.246.41'
# REDIS_HOST = '192.168.3.191'
REDIS_PORT = 6380

# mongodb相关配置
# MONGO_HOST = "172.16.13.165"
# # MONGO_HOST = "192.168.3.191"
# MONGO_PORT = 27017

# mysql相关配置
MYSQL_HOST = '218.28.2.162'
# MYSQL_HOST = '172.16.13.11'
MYSQL_DBNAME = 'bx_caiji'  # 数据库名
MYSQL_USER = 'caiji_user'
MYSQL_PASSWD = 'bxgj2019'
MYSQL_PORT = 3306
MYSQL_TABLE_KEYWORDS = 'bx_caiji_keywords'  # 关键字表
MYSQL_TABLE_KEYWORDS_NEWS = 'bx_caiji_keywords_news'  # 详情页信息表
# MYSQL_TABLE_KEYWORDS_NEWS = 'keyword_detail'  # 详情页信息表
